"""
Given a 10 or 13 digit ISBN fetches images from amazon if possible
-f file, takes a file listing one isbn per line

TODO: Why does future fetch get no markup for some queries? Threads clashing somehow?
TODO: Speed up url scraping - takes about 1sec to build soup currently

BUG: 
Processing:  9780415096225
list index out of range

TODO: Make more atomic ready for threads

"""

from BeautifulSoup import BeautifulSoup, SoupStrainer
import re
import urllib
import os
from os.path import exists, isfile
import sys
import getopt
import shelve
import future

class MissingMarkupWarning(Exception):
    def __init__(self, text=None):
        self.text = text

db = shelve.open('books.shelve')

import time

def print_timing(func):
    def wrapper(*arg):
        t1 = time.time()
        res = func(*arg)
        t2 = time.time()
        print '%s took %0.3f ms' % (func.func_name, (t2-t1)*1000.0)
        return res
    return wrapper


#@print_timing
def get_url_soup(url):
#    strained = SoupStrainer('span', id='btAsinTitle')
#    strained.search(markup)
#    print strained
    markup = get_markup_from_url(url)
    soup = BeautifulSoup(markup)
    return soup

def get_markup_from_url(url):
    u = urllib.urlopen(url)
    markup = u.read()
    u.close()
    if len(markup) < 10:
        raise MissingMarkupWarning()
    return markup

def get_title_from_soup(soup):
    item = soup.findAll('span', id='btAsinTitle')
    if not item:
        print 'Error - Missing tag span for title'
        return 'Unknown Title'
    title = item[0].string
    title = title.strip()
    return title
    
def get_author_from_soup(soup):
    item = soup.findAll('a', href=re.compile('field-author'))
    if not item:
        print 'Missing tag a for author'
        return 'Unknown Author'
    author = item[0].string
    author = author.strip()
    return author
    
def get_img_src_from_soup(soup):
    """
    Return the url for the image data
    """
    item = soup.findAll('img', id='prodImage')
    if not item:
        print "Error - Tag missing"
        return None
    img_src = re.compile('(http.*.jpg)').findall(str(item))
    if not img_src:
        print "Error - RegExp did not find JPG"
        return None
    img = re.compile('(http.*?_.*?_)').findall(img_src[0])
    if not img:
        print "Error - RegExp found JPG but did not recognise URL format"
        return None
    img_url = img[0] + '.jpg'
    return img_url

def get_data(isbn):
    """
    retrieve data from amazon for this ISBN
    @rtype: [title, author, img_src]
    """
    print "Fetching data for ISBN: ",  isbn
    search_url = 'http://www.amazon.co.uk/gp/search?keywords='+isbn+'&index=books'
    soup = get_url_soup(search_url)
    img_src = get_img_src_from_soup(soup)
    title = get_title_from_soup(soup)
    author = get_author_from_soup(soup)
    return [title, author, img_src]



# http://www.amazon.co.uk/exec/obidos/ISBN=0552134635


class DataFetch:
    def __init__(self, isbn):
        """ ISBN is 10 or 13 digit """
        self.isbn = isbn

    def get_data(self):
        """
        retrieve data from amazon for this ISBN
        @rtype: [title, author]
        """
        print "Fetching data for ISBN: ",  self.isbn
        search_url = 'http://www.amazon.co.uk/gp/search?keywords='+self.isbn+'&index=books'
        soup = get_url_soup(search_url)
        img_src = get_img_src_from_soup(soup)
        title = get_title_from_soup(soup)
        author = get_author_from_soup(soup)
        return [title, author, img_src]

    def get_image_url(self, soup):
        """
        Return the url for the image data
        """
        isbn = self.isbn
        try:
            item = soup.findAll('img', id='prodImage')
            if item:
                img_src = re.compile('(http.*.jpg)').findall(str(item))
                img = re.compile('(http.*?_.*?_)').findall(img_src[0])
                img_url = img[0] + '.jpg'
                return img_url
            else:
                return None
        except e:
            print e
            return None

    def fetch_image_to_file(self, url):
        try:
            local_file = self.download_dir+'/'+isbn+'.jpg'
            if exists(local_file):
                if isfile(local_file):
                    os.remove(local_file)
            urllib.urlretrieve(img_url, local_file)
            return True
        except Exception, e:
            print "Problem retrieving file: ", img_url
            print e
            return False


class ImageFetch:
    def __init__(self, isbn):
        """ ISBN is 10 or 13 digit """
        self.isbn = isbn
        self.download_dir = 'images'
        if not exists(self.download_dir):
            os.mkdir(self.download_dir)

#    @print_timing
    def get_image(self):
        if len(self.isbn) == 10:
            search_url = 'http://www.amazon.co.uk/gp/product/images/'+self.isbn+'/'
            soup = get_url_soup(search_url)
            if not self.get_image_from_amazon_image_html(soup):
                print "Error - no product image for ISBN-13: ", self.isbn
#            get_title_from_soup(soup)
        elif len(self.isbn) == 13:
            search_url = 'http://www.amazon.co.uk/gp/search?keywords='+self.isbn+'&index=books'
            soup = get_url_soup(search_url)
            if not self.get_image_from_amazon_search_html(soup):
                print "Error - no product image for ISBN-10: ", self.isbn
#            get_title_from_soup(soup)
        else:
            print "Error - isbn incorrect"

    def get_data(self):
        """
        retrieve data from amazon for this ISBN
        """
        global db
        search_url = 'http://www.amazon.co.uk/gp/search?keywords='+self.isbn+'&index=books'
        try:
            soup = get_url_soup(search_url)
        except Exception, e:
            print "Problem retrieving URL: ", search_url
            return
        if not self.get_image_from_amazon_search_html(soup):
            print "Warning - no product image for ISBN: ", self.isbn
        title = get_title_from_soup(soup)
        if not title:
            print "Warning - no title for ISBN: ", self.isbn
            title = 'Unknown Title'
        title=title.strip()

        author = get_author_from_soup(soup)
        if not author:
            print "Warning - no author for ISBN: ", self.isbn
            author = 'Unknown Author'
        author = author.strip()

        print title, " by ", author

        try:
            db[self.isbn] = [self.isbn, str(title), str(author)]
        except Exception, e:
            print "Problem storing: ", self.isbn, " in database"
            print e

    def get_image_from_amazon_image_html(self, soup):
        """
        For isbn 10 this is more direct (quicker)
        """
        isbn = self.isbn
        try:
            item = soup.findAll('img', id='prodImage')
            if item:
                img_src = re.compile('(http.*.jpg)').findall(str(item))
                img = re.compile('(http.*?_.*?_)').findall(img_src[0])
                img_url = img[0] + '.jpg'
                local_file = self.download_dir+'/'+isbn+'.jpg'
                if exists(local_file):
                    if isfile(local_file):
                        os.remove(local_file)
                urllib.urlretrieve(img_url, local_file)
                return True
            else:
                return False
        except e:
            raise e

    def get_image_from_amazon_search_html(self, soup):
        """
        The most robust way of finding the correct book and image. But returns lots of html so it's slow.
        """
        isbn = self.isbn
        try:
            item = soup.findAll('img', id='prodImage')
            if item:
                img_src = re.compile('(http.*.jpg)').findall(str(item))
                img = re.compile('(http.*?_.*?_)').findall(img_src[0])
                img_url = img[0] + '.jpg'
                local_file = self.download_dir+'/'+isbn+'.jpg'
                if exists(local_file):
                    if isfile(local_file):
                        os.remove(local_file)
                urllib.urlretrieve(img_url, local_file)
                return True
            else:
                return False
        except Exception, e:
            raise e

def is_valid_isbn(isbn):
    if len(isbn) == 10 or len(isbn) == 13:
        return True
    return False 

def process_isbn(isbn):
    global db
    if not isbn in db:
        if is_valid_isbn(isbn):
            print "Processing: ", isbn
            f = ImageFetch(isbn)
    #        f.get_image()
            f.get_data()
        else:
            print "Error - not an isbn: ", isbn
    else:
        print "Skipping: ", isbn, " as already in db"


def process_isbn_file(filename):
    """
    Processes a file filled with 10 or 13 digit isbns. 
    """
    try:
        file = open(filename)
        for isbn in file:
            isbn = isbn.strip('\n')
            try:
                process_isbn(isbn)
            except Exception, e:
                print e
    except Exception, e:
        print e
        return 2
    return 0
    

class Usage(Exception):
    def __init__(self, msg):
        self.msg = msg

def main(argv=None):
    if argv is None:
        argv = sys.argv
    try:
        try:
            opts, args = getopt.getopt(argv[1:], "hf:", ["help", "file"])
        except getopt.error, msg:
             raise Usage(msg)

        # process options
        for o, a in opts:
            if o in ("-h", "--help"):
                print __doc__
                return 0
            if o in ("-f", "--file"):
                process_isbn_file(a)
                return 0


        # process arguments, isbns on the command line
        for arg in args:
            process_isbn(arg) # process() is defined elsewhere


    except Usage, err:
        print >>sys.stderr, err.msg
        print >>sys.stderr, "for help use --help"
        return 1

def fetch_data(isbn):
    """
    Fetch the data.
    If problems occur due that could be solved by retrying - indicate this 
    """
    res = None
    try:
        res = get_data(isbn)
    except MissingMarkupWarning, e:
        res = 'MissingMarkup'
    return res

@print_timing
def future_fetch():
    futures=[]
    file = open('isbn3.txt')
    for isbn in file:
        isbn = isbn.strip('\n')
        futures.append([isbn, future.Future(fetch_data, isbn)])
        time.sleep(0.5) # don't spam amazon's servers or they return nothing and we have to retry
    count=0

    while(len(futures)):
        isbn, a = futures.pop(0)
        res = a()
        print count, ": ", isbn, ": ", res
        count += 1
        if res == None:
            print isbn, 'No result'
        t = type(res)
        if t == list:
            print isbn, res
        elif t == str:
            if res == 'MissingMarkup':
                print isbn, ': Markup was missing - retrying'
                futures.append([isbn, future.Future(fetch_data, isbn)])
                

@print_timing
def linear_fetch():
    file = open('testisbn.txt')
    for isbn in file:
        isbn = isbn.strip('\n')
        print fetch_data(isbn)
    

if __name__ == "__main__":
#    sys.exit(main())
    future_fetch()
#    linear_fetch()
#    future_fetch()
#    linear_fetch()
       
        

